package first;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.StringTokenizer;


public class NoiseFilter 
{
	private ArrayList<String> noise = new ArrayList<String>();
	private ArrayList<Tweet> tweets;
	private StringTokenizer st;
	
	public NoiseFilter()
	{
		this.read();
	}
	
	/*public ArrayList<Tweet> removeNoise1(ArrayList<Tweet> tweets2)
	{
		this.tweets = tweets2;
		
		for(int i = 0; i < tweets.size(); i++)
        {
        	String text = tweets.get(i).getText().toLowerCase();
        	
        	for(int j = 0; j < noise.size(); j++)
        	{
        		text.replace(noise.get(i), "");
        	}
        	
        	tweets.get(i).setTextSent(text);
        }
		
		return tweets;
	}*/
	
	public ArrayList<Tweet> removeNoise2(ArrayList<Tweet> tweets2)
	{
		this.tweets = tweets2;
		
		String text = "";
		String token = "";
		
		for(int i = 0; i < tweets.size(); i++)
	    {
	    	this.st = new StringTokenizer(tweets.get(i).getText().toLowerCase());
	    	
	    	while(st.hasMoreTokens())
	    	{
	    		token = st.nextToken();
	    		token.replaceAll("[^a-zA-Z]", "");
	    		
	    		if(token.length() < 3 || token.contains("<") || token.contains("=") || noise.contains(token) || text.contains(token))
	        	{
	        		// Noise word or already in the text
	        	}
	        	else
	        	{
	        		text += (" "+token);
	        	}
	    	}
	    	
	    	tweets.get(i).setTextSent(text);
	    	
	    	text = "";
	    	token = "";
	    }
		return tweets;
	}
	
	public void read()
	{
		try 
		{
			BufferedReader in = new BufferedReader(new FileReader("noise.txt"));
			String line = null;
			while ((line = in.readLine()) != null) 
			{
				noise.add(line);
			}
		} 
		catch (IOException e) 
		{
			e.printStackTrace();
		}
	}
}
